package us.codecraft.webmagic.pipeline;

import java.net.UnknownHostException;

import org.apache.commons.codec.digest.DigestUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Task;
import us.codecraft.webmagic.utils.FilePersistentBase;

import com.mongodb.BasicDBObject;
import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.Mongo;
import com.mongodb.MongoException;

public class MongoPipeline extends FilePersistentBase implements Pipeline   {
	DB db;
	DBCollection coll;
	DBCollection coll_url;
	Mongo mc;
	
	public MongoPipeline(String db,String coll) {
		try {
			this.mc = new Mongo();
		} catch (UnknownHostException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (MongoException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		this.db = mc.getDB(db);
        this.coll = this.db.getCollection(coll);
        this.coll_url = this.db.getCollection("jtest_url");
    }
    
	@Override
	public void process(ResultItems resultItems, Task task) {
		// TODO Auto-generated method stub
		BasicDBObject mongoDoc = new BasicDBObject();
		BasicDBObject urlHashTable = new BasicDBObject();
		java.util.Date date= new java.util.Date();
				
		Document doc = Jsoup.parse(resultItems.getAll().entrySet().toString());
		mongoDoc = new HTMLToJSON(Jsoup.parse(doc.toString())).ElementsToJSON();
		mongoDoc.append("time", date.getTime());
		mongoDoc.append("url", resultItems.getRequest().getUrl());
		mongoDoc.append("url_hash", DigestUtils.md5Hex(resultItems.getRequest().getUrl()));
		coll.insert(mongoDoc);

		urlHashTable.append("time", date.getTime());
		urlHashTable.append("url", resultItems.getRequest().getUrl());
		urlHashTable.append("url_hash", DigestUtils.md5Hex(resultItems.getRequest().getUrl()));
		coll_url.insert(urlHashTable);
	}

}
